{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 分会议建立bulk_json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import read_pdf, listdir, org_years_dict\n",
    "import json, os\n",
    "from tqdm import tqdm_notebook \n",
    "from multiprocessing import Pool\n",
    "import time, traceback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18810"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file_paths = []\n",
    "listdir('/home/d/papers_data/', file_paths)\n",
    "len(file_paths)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# bulk insert!(ok)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "! curl -X DELETE \"localhost:9200/acl?pretty\"\n",
    "! curl -X DELETE \"localhost:9200/aaai?pretty\"\n",
    "! curl -X DELETE \"localhost:9200/emnlp?pretty\"\n",
    "! curl -X DELETE \"localhost:9200/icml?pretty\"\n",
    "! curl -X DELETE \"localhost:9200/iclr?pretty\"\n",
    "! curl -X DELETE \"localhost:9200/kdd?pretty\"\n",
    "! curl -X DELETE \"localhost:9200/nips?pretty\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "health status index uuid                   pri rep docs.count docs.deleted store.size pri.store.size\n",
      "yellow open   iclr  wdQPpCI5TFq1uGwUslVj_w   1   1       1980            8    178.2mb        178.2mb\n",
      "yellow open   bank  U8Pq9fF_SO-qmNwuVGYzLg   1   1       1000            0    414.3kb        414.3kb\n",
      "yellow open   aaai  cjDr4muCRA-_MXyxSbSjag   1   1       5062            8    213.9mb        213.9mb\n",
      "yellow open   emnlp hqBbqnSuT82-ZFPhixb-4A   1   1       3245            1    101.9mb        101.9mb\n",
      "yellow open   icml  RihCp80hTEu_K9ij7UEPkw   1   1       1054            0       42mb           42mb\n",
      "yellow open   kdd   iRGYz-AMR_-YmL15Oa4OAA   1   1        828            5     41.8mb         41.8mb\n",
      "yellow open   acl   gCWVLshARBm0lsQe8fUVNA   1   1       3434            2    107.9mb        107.9mb\n",
      "yellow open   nips  ClZZkJ0RQZ21EARwV8fO_A   1   1       3114            7      114mb          114mb\n"
     ]
    }
   ],
   "source": [
    "! curl -X GET \"localhost:9200/_cat/indices?v\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18717"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "1980+5062+3245+1054+ 828+3434+3114\n",
    "# 应该要是18724-3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "2+0+2+0+5+1+8+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['./pdf_es_json/ACL_2014_2015_2016_2017_2018_2019.json',\n",
       " './pdf_es_json/AAAI_2014_2015_2016_2017_2018_2019.json',\n",
       " './pdf_es_json/EMNLP_2014_2015_2016_2017_2018_2019.json',\n",
       " './pdf_es_json/ICLR_2017_2018_2019.json',\n",
       " './pdf_es_json/ICML_2017_2018.json',\n",
       " './pdf_es_json/KDD_2017_2018_2019.json',\n",
       " './pdf_es_json/NIPS_2017_2018_2019.json']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from utils import org_years_dict\n",
    "json_file_folder = './pdf_es_json'\n",
    "json_file_paths = []\n",
    "for org in org_years_dict.keys():\n",
    "    file_name = json_file_folder + f'/{org}_' + '_'.join([year for year in org_years_dict[org]]) + '.json'\n",
    "    json_file_paths.append(file_name)\n",
    "\n",
    "json_file_paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json, requests\n",
    "from utils import org_years_dict\n",
    "\n",
    "def bulk_insert2ES(org, bulk_size = 300, json_file_folder = './pdf_es_json'):\n",
    "    json_file_paths = []\n",
    "    for _org in org_years_dict.keys():\n",
    "        file_name = json_file_folder + f'/{_org}_' + '_'.join([year for year in org_years_dict[_org]]) + '.json'\n",
    "        json_file_paths.append(file_name)\n",
    "    \n",
    "    for path in json_file_paths:\n",
    "        if path.split('/')[-1].split('_')[0] != org: continue\n",
    "    \n",
    "        with open(path, 'r') as f1:\n",
    "            bulk_content_lines = f1.readlines()\n",
    "        \n",
    "        url = f'http://localhost:9200/{org.lower()}/_bulk?pretty&refresh'\n",
    "        headers = {'Content-Type': 'application/json'}\n",
    "        for _index in range(0, len(bulk_content_lines), bulk_size*2):\n",
    "            data = ''.join(bulk_content_lines[_index: _index + bulk_size*2]) + '\\n'\n",
    "#             print(len(data))\n",
    "            r = requests.post(url = url, headers = headers, data=data)\n",
    "#             j = json.loads(r.text)\n",
    "#             print(j)\n",
    "#             return j\n",
    "#             break\n",
    "    print(f'{org} DONE!')\n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ACL DONE!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bulk_insert2ES(org = 'ACL', bulk_size = 300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EMNLP DONE!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bulk_insert2ES(org = 'EMNLP', bulk_size = 300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AAAI DONE!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bulk_insert2ES(org = 'AAAI', bulk_size = 300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ICLR DONE!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bulk_insert2ES(org = 'ICLR', bulk_size = 300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ICML DONE!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bulk_insert2ES(org = 'ICML', bulk_size = 300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KDD DONE!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bulk_insert2ES(org = 'KDD', bulk_size = 300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NIPS DONE!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bulk_insert2ES(org = 'NIPS', bulk_size = 300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 查询(失败)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _base_script(key_word:str):\n",
    "    base_script = {}\n",
    "    base_script[\"key_words_frequency\"] = {}\n",
    "    base_script[\"key_words_frequency\"][\"script\"] = {}\n",
    "    base_script[\"key_words_frequency\"][\"script\"][\"lang\"] = \"painless\"\n",
    "    base_script[\"key_words_frequency\"][\"script\"][\"source\"] = f\"\"\"\n",
    "    int count = 0;\n",
    "    if(doc['Text.keyword'].size() > 0 && doc['Text.keyword'].value.indexOf(params.phrase)!=-1) count++;\n",
    "    return count;        \n",
    "    \"\"\"\n",
    "    base_script[\"key_words_frequency\"][\"script\"][\"params\"] = {}\n",
    "    base_script[\"key_words_frequency\"][\"script\"][\"params\"][\"phrase\"] = key_word\n",
    "\n",
    "    return base_script"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'key_words_frequency': {'script': {'lang': 'painless',\n",
       "   'source': \"\\n    int count = 0;\\n    if(doc['Text.keyword'].size() > 0 && doc['Text.keyword'].value.indexOf(params.phrase)!=-1) count++;\\n    return count;        \\n    \",\n",
       "   'params': {'phrase': 'represent'}}}}"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = _base_script('represent')\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'took': 4,\n",
       " 'timed_out': False,\n",
       " '_shards': {'total': 1, 'successful': 1, 'skipped': 0, 'failed': 0},\n",
       " 'hits': {'total': {'value': 3495, 'relation': 'eq'},\n",
       "  'max_score': None,\n",
       "  'hits': [{'_index': 'acl',\n",
       "    '_type': '_doc',\n",
       "    '_id': '000a817558bcecf98cdf4c991a5852d5',\n",
       "    '_score': None,\n",
       "    '_source': {'Year': '2014',\n",
       "     'Name': 'A practical and linguistically-motivated approach to compositional distributional semantics'},\n",
       "    'fields': {'key_words_frequency': [0]},\n",
       "    'sort': ['000a817558bcecf98cdf4c991a5852d5']}]}}"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 统计特定词语的词频\n",
    "data = {\n",
    "    \"_source\": {\n",
    "        \"include\": [\n",
    "            \"Name\",\n",
    "            \"Year\"\n",
    "            ]\n",
    "            },\n",
    "    \"from\":0,\n",
    "    \"size\":1,\n",
    "    \"sort\": { \"_id\": { \"order\": \"asc\" }},\n",
    "  \"script_fields\": x\n",
    "}\n",
    "headers = {'Content-Type': 'application/json'}\n",
    "\n",
    "r = requests.get(\"http://localhost:9200/acl/_search\", headers = headers, data = json.dumps(data))\n",
    "json.loads(r.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "j = json.loads(r.text)\n",
    "lines = []\n",
    "jj = j['hits']['hits']\n",
    "for item in jj:\n",
    "    one_line = []\n",
    "    address = item['_source']['address']\n",
    "    balance = item['_source']['balance']\n",
    "    temp = [item['fields'][key][0] for key in item['fields'].keys()]\n",
    "    one_line.append(address)\n",
    "    one_line.append(balance)\n",
    "    one_line.extend(temp)\n",
    "    lines.append(one_line)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 查找特定字段\n",
    "data = {\n",
    "  \"query\": { \"match\": { \"Text\": \"approch\" } }\n",
    "}\n",
    "headers = {'Content-Type': 'application/json'}\n",
    "\n",
    "r = requests.get(\"http://localhost:9200/acl/_search\", headers = headers, data = json.dumps(data))\n",
    "json.loads(r.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 失误处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3114\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "with open('./pdf_es_json/NIPS_2017_2018_2019.json', 'r') as f1:\n",
    "    rr = json.load(f1)\n",
    "    print(len(rr))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('./pdf_es_json/NIPS_2017_2018_2019.json', 'w') as f2:\n",
    "    for i in rr:\n",
    "        f2.write(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"index\":{}}\n"
     ]
    }
   ],
   "source": [
    "! head -n 1 ./pdf_es_json/NIPS_2017_2018_2019.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 手动处理错误\n",
    "\n",
    "```\n",
    "/home/d/papers_data/ACL/ACL_2018/Do\\ Neural\\ Network\\ Cross-Modal\\ Mappings\\ Really\\ Bridge\\ Modalities\\ .pdf\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7007\n",
      "9985\n",
      "3967\n",
      "2107\n",
      "1633\n",
      "6229\n"
     ]
    }
   ],
   "source": [
    "# 加md5处理; 将dumps文件转为ES bulk文件\n",
    "from utils import curlmd5\n",
    "import json\n",
    "\n",
    "from utils import org_years_dict\n",
    "json_file_folder = './pdf_es_json/old/'\n",
    "json_file_paths = []\n",
    "for org in org_years_dict.keys():\n",
    "    if org == 'EMNLP': continue\n",
    "    file_name = json_file_folder + f'{org}_' + '_'.join([year for year in org_years_dict[org]]) + '.json'\n",
    "    json_file_paths.append(file_name)\n",
    "\n",
    "for path in json_file_paths:\n",
    "    \n",
    "    with open(path, 'r') as f1:\n",
    "        _content = f1.read().split('\\n')\n",
    "        print(len(_content))\n",
    "\n",
    "    for i in range(0, len(_content)-1, 2):\n",
    "        try:\n",
    "            _name = json.loads(_content[i+1])[\"Name\"]\n",
    "            _new_start = '{\"index\":{\"_id\":\\\"' + curlmd5(_name) + '\\\"}}'\n",
    "            _content[i] = _new_start\n",
    "        except:\n",
    "            print(i)\n",
    "            print(_content[i+1])\n",
    "\n",
    "    pdf_name = path.split('/')[-1]\n",
    "    with open(f'./pdf_es_json/{pdf_name}', 'w') as f2:\n",
    "        for i in _content:\n",
    "            f2.write(i+'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 个别pdf无法读取; 设置超时时间\n",
    "from pdfminer.layout import LAParams\n",
    "from pdfminer.high_level import extract_text\n",
    "laparams=LAParams(\n",
    "    line_overlap=0.5,\n",
    "    char_margin=2.0,\n",
    "    line_margin=0.5,\n",
    "    word_margin=0.1,\n",
    "    boxes_flow=0.5,\n",
    "    detect_vertical=True,\n",
    "    all_texts=False,\n",
    ")"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "extract_text('/home/d/papers_data/EMNLP/EMNLP_2015/Online Updating of Word Representations for Part-of-Speech Tagging.pdf',laparams=laparams)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://pypi.tuna.tsinghua.edu.cn/simple\n",
      "Collecting func_timeout\n",
      "\u001b[?25l  Downloading https://pypi.tuna.tsinghua.edu.cn/packages/b3/0d/bf0567477f7281d9a3926c582bfef21bff7498fc0ffd3e9de21811896a0b/func_timeout-4.3.5.tar.gz (44kB)\n",
      "\u001b[K     |████████████████████████████████| 51kB 3.0MB/s eta 0:00:011\n",
      "\u001b[?25hBuilding wheels for collected packages: func-timeout\n",
      "  Building wheel for func-timeout (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for func-timeout: filename=func_timeout-4.3.5-cp37-none-any.whl size=15078 sha256=30e0fadaa26e089b4429b19f92411f5741a4fd435edbdb1a74a7c9fe7e5ca3bf\n",
      "  Stored in directory: /root/.cache/pip/wheels/f6/17/26/63fc3d4477af2658d29baacf3279bafdf85222eae909744f32\n",
      "Successfully built func-timeout\n",
      "Installing collected packages: func-timeout\n",
      "Successfully installed func-timeout-4.3.5\n"
     ]
    }
   ],
   "source": [
    "!pip install func_timeout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 设置超时功能\n",
    "from func_timeout import func_timeout, FunctionTimedOut\n",
    "\n",
    "def _time_out_wrap(do_function, time_out:int , *kwargs):\n",
    "    try:\n",
    "        doitReturnValue = func_timeout(time_out, do_function, args = kwargs)\n",
    "        return doitReturnValue\n",
    "    except FunctionTimedOut:\n",
    "        print ( f\"{do_function.__name__}{kwargs} could not complete within {time_out} seconds and was terminated.\")\n",
    "    except:\n",
    "        traceback.print_exc()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "def test(x1, x2):\n",
    "    time.sleep(4)\n",
    "    return f'oook {x1} {x2}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test ('22', '222') could not complete within 3 seconds and was terminated.\n"
     ]
    }
   ],
   "source": [
    "_time_out_wrap(test, 3, '22', '222')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'test'"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.__name__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "322\n"
     ]
    }
   ],
   "source": [
    "path = './test.json'\n",
    "\n",
    "with open(path, 'r') as f1:\n",
    "    bulk_content_lines = f1.readlines()\n",
    "\n",
    "url = f'http://localhost:9200/test/_bulk?pretty&refresh'\n",
    "headers = {'Content-Type': 'application/json'}\n",
    "for _index in range(0, len(bulk_content_lines), 2):\n",
    "    data = ''.join(bulk_content_lines[_index: _index + 2]) + '\\n'\n",
    "    print(len(data))\n",
    "    r = requests.post(url = url, headers = headers, data=data)\n",
    "    j = json.loads(r.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'took': 20,\n",
       " 'errors': False,\n",
       " 'items': [{'index': {'_index': 'test',\n",
       "    '_type': '_doc',\n",
       "    '_id': '7613d44ff8ad22f8a1a86612b4b40c61',\n",
       "    '_version': 3,\n",
       "    'result': 'updated',\n",
       "    'forced_refresh': True,\n",
       "    '_shards': {'total': 2, 'successful': 1, 'failed': 0},\n",
       "    '_seq_no': 2,\n",
       "    '_primary_term': 1,\n",
       "    'status': 200}}]}"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "j"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _base_script(key_word:str):\n",
    "    base_script = {}\n",
    "    base_script[\"key_words_frequency\"] = {}\n",
    "    base_script[\"key_words_frequency\"][\"script\"] = {}\n",
    "    base_script[\"key_words_frequency\"][\"script\"][\"lang\"] = \"painless\"\n",
    "    base_script[\"key_words_frequency\"][\"script\"][\"source\"] = f\"\"\"\n",
    "    int count = 0;\n",
    "    if(doc['Text.keyword'].size() > 0 && doc['Text.keyword'].value.indexOf(params.phrase)!=-1) count++;\n",
    "    return count;        \n",
    "    \"\"\"\n",
    "    base_script[\"key_words_frequency\"][\"script\"][\"params\"] = {}\n",
    "    base_script[\"key_words_frequency\"][\"script\"][\"params\"][\"phrase\"] = key_word\n",
    "\n",
    "    return base_script"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'key_words_frequency': {'script': {'lang': 'painless', 'source': \"\\n    int count = 0;\\n    if(doc['Text.keyword'].size() > 0 && doc['Text.keyword'].value.indexOf(params.phrase)!=-1) count++;\\n    return count;        \\n    \", 'params': {'phrase': 'embeddings'}}}}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'took': 2,\n",
       " 'timed_out': False,\n",
       " '_shards': {'total': 1, 'successful': 1, 'skipped': 0, 'failed': 0},\n",
       " 'hits': {'total': {'value': 1, 'relation': 'eq'},\n",
       "  'max_score': None,\n",
       "  'hits': [{'_index': 'test',\n",
       "    '_type': '_doc',\n",
       "    '_id': '7613d44ff8ad22f8a1a86612b4b40c61',\n",
       "    '_score': None,\n",
       "    '_source': {},\n",
       "    'fields': {'key_words_frequency': [1]},\n",
       "    'sort': ['7613d44ff8ad22f8a1a86612b4b40c61']}]}}"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 统计特定词语的词频\n",
    "x = _base_script(\"embeddings\")\n",
    "print(x)\n",
    "data = {\n",
    "    \"_source\": {\n",
    "        \"include\": [\n",
    "            \"address\",\n",
    "            ]\n",
    "            },\n",
    "    \"from\":0,\n",
    "    \"size\":10,\n",
    "    \"sort\": { \"_id\": { \"order\": \"asc\" }},\n",
    "  \"script_fields\": x\n",
    "}\n",
    "headers = {'Content-Type': 'application/json'}\n",
    "\n",
    "r = requests.get(\"http://localhost:9200/test/_search\", headers = headers, data = json.dumps(data))\n",
    "json.loads(r.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
